<?php
require_once __DIR__.'/rssCollector.php';
class PeopleRssCollector extends rssCollector{
    
    //构造函数，初始化资源列表和上次抓取时间
    public function __construct() {
		$this->dataPath = storage_path().'/collector/peopleData.php';
        $this->rssSourceMap = $this->getCategory();
		$this->categoryMap = $this->getCategoryMap();
        $this->lastGetTime = $this->getLastGetTime();
    }
    
    //执行抓取
    public function execut() {
        $beginTotalTime = microtime(true);
        //每个大类依次处理
        foreach ($this->rssSourceMap as $category){
			//匹配出频道
			preg_match("/ *http:\/\/([a-xA-Z.]*?)\.people\.com\.cn.*/is", $category['htmlUrl'], $matches);
			$channel = isset($matches[1]) ? trim($matches[1]) : ''; 
            //获取次类列表,不包含详细内容，关键词等
            $listSimple = $this->getList($category['xmlUrl'], $category['title'], $channel);
            //根据频道，使用相应的方式处理具体内容
            $listDetail = $this->getDetail($listSimple);
            //保存至数据库
            $beginSaveTime = microtime(true);
            $this->saveToDB($listDetail);
            $endSaveTime = microtime(true);
            $this->saveTime = $endSaveTime - $beginSaveTime;
        }
        $endTotalTime = microtime(true);
        $this->totalTime = $endTotalTime - $beginTotalTime;
        $this->setLastGetTime();
    }
    
    //获取给定分类的简单列表
    private function getList($xmlUrl, $categoryName, $channel = ''){
        $beginTransferTime = microtime(true);
		$listXML = $this->simpleRequest($xmlUrl);
        $endTransferTime = microtime(true);
        $this->transferTime += $endTransferTime - $beginTransferTime;
        //结果过滤+格式整理
        $beginParseTime = microtime(true);
        $newsListArrRaw = $this->convertXMLToArray($listXML);
		$newsListArr = isset($newsListArrRaw['channel']['item']) ? $newsListArrRaw['channel']['item'] : array();
		foreach ($newsListArr as $key => $newsSimple){
			//从数组删除数据
			unset($newsListArr[$key]);
			//没有日期的直接放弃
			if(!isset($newsSimple['pubDate']) || !$newsSimple['pubDate']){
				$this->matchErrorCount += 1;
				continue;
			}
			$newsItem = array();
			$newsPubTime = $newsItem['pubDate'] = strtotime($newsSimple['pubDate']);
			//忽略已经抓过的
			if($newsPubTime < $this->lastGetTime){
				continue;
			}
			//唯一标识
			$newsItem['news_id'] = 'people_'.$newsSimple['news_id'];
			//标记来源
			$newsItem['source'] = 'people';
			//作者
			$newsItem['author'] = 'people';
			//链接处理
			$newsItem['link'] = $newsLink = $newsSimple['link'];
			//url为空的直接放弃
			if(!$newsLink){
				$this->matchErrorCount += 1;
				continue;
			}
			//匹配出频道
			$newsItem['channel'] = $channel; 
			//分类
			$newsItem['category'] = $this->getLocalCategory($categoryName, 12);
			//标题格式化
			$newsItem['title'] = preg_replace("/\s+/","",$newsSimple['title']);
			$newsListArr[$key] = $newsItem;
		}
        $endParseTime = microtime(true);
        $this->parseTime += $endParseTime - $beginParseTime;
        return $newsListArr;
    }
    
    //获取详细内容
    private function getDetail($newsListArr){
		//组装抓取节点
		$nodes = array();
		foreach ($newsListArr as $key => $newsArr){
			$nodes[$key] = $newsArr['link'];
		}
		//获取列表内容
		$beginTransferTime = microtime(true);
		$newsListContent = $this->multipleThreadsRequest($nodes);
		$endTransferTime = microtime(true);
		$this->transferTime += $endTransferTime - $beginTransferTime;
		//匹配内容和关键词
		$beginParseTime = microtime(true);
		foreach ($newsListArr as $key => $value){
			$content = mb_convert_encoding($newsListContent[$key], 'UTF-8', 'GB2312');
			$newsListArr[$key]['slides'] = '';
			$newsListArr[$key]['video'] = '';
			$newsListArr[$key]['keywords'] = $this->matchKeywords($content); 
			$newsListArr[$key]['description'] = $this->matchDescription($content);			
			//将图片的相对地址改成绝对地址
			$contentDetail = $this->imgRelToAbs($this->matchContnet($content, $nodes[$key]), $nodes[$key]);
			if(!$contentDetail){
				unset($newsListArr[$key]);
				$this->matchErrorCount += 1;
				continue;
			}
			$newsListArr[$key]['content'] = $contentDetail;
			//提取图片
			$imgs = $this->matchImgs($contentDetail);
			$newsListArr[$key]['img'] = isset($imgs[0]) ? $imgs[0] : '';
			$newsListArr[$key]['imgs'] = implode(';', $imgs);
		}
		$endParseTime = microtime(true);
		$this->parseTime += $endParseTime - $beginParseTime;
		return $newsListArr;
    }
    
    //匹配内容
    private function matchContnet($content, $url = '', $matchPagenate = false){
		//获取内容片断
		$content = $this->matchDiv($content, '/<div(?:.*?)id="p_content"(?:.*?)>/i');
		//获取分页内容
		if(!$matchPagenate){
			$content = $this->matchPagenateContent($content, $url);
		}
		return $content;
    }
	
	//匹配分页内容（拼接）
	private function matchPagenateContent($content, $url){
		//获取分页div
		$pagenation = $this->matchDiv($content, '/<div class="zdfy clearfix">/i');
		if(!$pagenation){
			//去广告
			$otitle = $this->matchDiv($content, '/<div class="otitle">/i');
			if($otitle){
				$content = str_replace('<div class="otitle">'.$otitle.'</div>', '', $content);
			}
			$textcode = $this->matchDiv($content, '/<div class="textcode">/i');
			if($textcode){
				$content = str_replace('<div class="textcode">'.$textcode.'</div>', '', $content);
			}
			return $content;
		}
		//去分页
		$content = str_replace('<div class="zdfy clearfix">'.$pagenation.'</div>', '', $content);
		$content = preg_replace('/<center>(?:.*?)<\/center>/i', '', $content);
		//获取分页连接
		$parten = '/<a(?:.*?)href="(.*?)".*?>(.*?)<\/a>/i';
		preg_match_all($parten, $pagenation, $matches);
		$links = isset($matches[1]) ? array_unique($matches[1]) : array();
		if(!$links){
			return $content;
		}
		//匹配出相对当前url的图片路径的前缀
		preg_match('/.*\//i', $url, $matches);
		$urlDirCurrent = isset($matches[0]) ? $matches[0] : '';
		//匹配出相对根目录的图片地址前缀
		preg_match('/http:\/\/.*?\//i', $url, $matches);
		$urlDirRoot = isset($matches[0]) ? substr($matches[0], 0, -1) : '';
		//组装抓取节点
		$nodes = array();
		foreach ($links as $key => $link){
			$realUrl = '';
			if(strpos($link, 'http://') === 0){//绝对图片地址
				$realUrl = $link;
			}elseif(strpos($link, '/') === 0){//相对于根目录
				$realUrl = $urlDirRoot.$link;
			}else{//相对于当前网页
				$realUrl = $urlDirCurrent.$link;
			}
			$nodes[$key] = $realUrl;
		}
		//获取列表内容
		$beginTransferTime = microtime(true);
		$newsListContent = $this->multipleThreadsRequest($nodes);
		$endTransferTime = microtime(true);
		$this->transferTime += $endTransferTime - $beginTransferTime;
		$content = '';
		foreach ($newsListContent as $value){
			$value = mb_convert_encoding($value, 'UTF-8', 'GB2312');
			$content_temp = $this->matchContnet($value, $url, true);
			$pagenation_temp = $this->matchDiv($content_temp, '/<div class="zdfy clearfix">/i');
			//去分页
			$content_temp = str_replace('<div class="zdfy clearfix">'.$pagenation_temp.'</div>', '', $content_temp);
			$content_temp = preg_replace('/<center>(?:.*?)<\/center>/i', '', $content_temp);
			//去广告
			$otitle = $this->matchDiv($content_temp, '/<div class="otitle">/i');
			if($otitle){
				$content_temp = str_replace('<div class="otitle">'.$otitle.'</div>', '', $content_temp);
			}
			$textcode = $this->matchDiv($content_temp, '/<div class="textcode">/i');
			if($textcode){
				$content_temp = str_replace('<div class="textcode">'.$textcode.'</div>', '', $content_temp);
			}
			$content .= $content_temp;
		}
		return $content;
	}
	
	//匹配关键词
    private function matchKeywords($content){
        $matches = array();
        preg_match('/<meta name="keywords" content="(.*?)"/is', $content, $matches);
        $resHtml = isset($matches[1]) ? trim($matches[1]) : '';
		$keywordsArr = explode(' ', $resHtml);
        return $keywordsArr ? implode(',', $keywordsArr) : '';
    }
	
	//匹配简介
    private function matchDescription($content){
        $matches = array();
        preg_match('/<meta name="description" content="(.*?)"/is', $content, $matches);
        $resHtml = isset($matches[1]) ? trim($matches[1]) : '';
        return $resHtml;
    }
	
	//刷新人民网rss源
	public function flushCategoryMap(){
		$url = 'http://www.people.com.cn/rss/opml.xml';
		$res = $this->multipleThreadsRequest(array(0 => $url));
		$newsListArrRaw = $this->convertXMLToArray($res[0]);
		$itemsRaw = $newsListArrRaw['body']['outline'];
		$items = array();
		foreach ($itemsRaw as $key => $value) {
			$items[] = $value['@attributes'];
		}
		$data = require $this->dataPath;
		$data['category'] = $items;
		$data_str = '<?php return '.var_export($data, true).';';
        return file_put_contents($this->dataPath, $data_str);
	}
}